package nci60.query
import nci60.data.protein.Protein
import nci60.data.maxquant.Project
import nci60.data.maxquant.Experiment
import nci60.data.maxquant.ExperimentClass
import nci60.data.connected.LeadingProtein
import java.text.DecimalFormat
import grails.converters.JSON

/**
 * GRAILS <a href="http://grails.org/doc/latest/guide/theWebLayer.html#controllers" target="_blank">Controllers</a>
 * This controller handles all actions related to querying the database for a list of proteins. <p />
 * This addresses searching for proteins or the global expression analysis, including all associated atcions like initialization, generating the results and downloading the results.
 * It also includes the generation of the suggestions for the autocomplete function (also for the main page).
 *
 * @autor <a href="mailto:Florian.J.Auer@googlemail.com">Florian J. Auer</a>
*/
class QueryController {

    /**
     * Makes the Query Service accessable
     * @see helper.QueryService
     */
    def queryService
    
    /**
     * Makes the Helper Service accessable
     * @see helper.HelperService
     */
    def helperService

    /**
     * redirects to the proteins action (protein search page) as default entry point of this controller
     */
    def index = {
        redirect(action:'proteins')
    }
    
    /**
     * Entry action for "Global expression analysis" search option <p />
     * This gives necessary information for the form (tissues of origin as celllines). <p />
     * Further: <p />
     * The suggestions are generated by {@link #suggest}, called via AJAX, and rendered as JSON <p />
     * The global expression analysis is performed by {@link #globalExpressionResults} for the web rendering and {@link #globalExpressionDownload} for generating a file for download.
     */
    def globalExpression = {
        def celllines = []
        ExperimentClass.executeQuery('select distinct c.name from ExperimentClass c').each{cls->
            celllines.add(cls)
        }
        return [Celllines:celllines]
    }

    
    /**
     * Generates the suggestions for the search boxes in main page, proteins and global expression analysis <p />
     * The action is called via AJAX and the results are rendered in JSON format, so that it can be read by select2<p />
     * <p />
     * 
     * Parameter (params): <p />
     * 
     * <style>.custom-table td{border-top:1pt solid grey;vertical-align:top;} .custom-table{border-collapse:collapse;width:100%;} .custom-table .z{background-color:rgb(230, 230, 230);}</style>
     * <table class="custom-table">
     * <tr><td class="z">
     *              query
     * </td><td>        user input for which the suggestions should be provided <p />
     *                  this method only is called if length of query is 3 or greater
     * </td></tr>
     * <tr><td class="z">
     *              max
     * </td><td>        number of maximal shown entries in the list of suggestions <p />
     *                  is set by initializing select2 (home.js for main page, find.js for "Proteins" or search.js for "Global expression analysis"
     * </td></tr>
     * <tr><td class="z">
     *              offset
     * </td><td>        offset for the list of suggestions; given by select2
     * </td></tr>
     * </table>
     * 
     * <p />
     * <p />
     * 
     * Result:
     * <p />
     * 
     * <table class="custom-table">
     * <tr><td class="z">      .elements[]
     * </td><td>                     .type=
     * </td><td class="z">                  type of the element <p />
     *                                      used for defining the class of the element in the list as well as for generating the query: <p />
     *                                      'gene' for gene name, 'uniprot' for protein name, 'go' for GO, 'ipi' for IPI identifier, 'description' for protein description
     * </td></tr>
     * <tr><td class="z"> 
     * </td><td>                     .name=
     * </td><td class="z">                  name of the elemnet <p />
     *                                      shown in list and if selected
     * </td></tr>
     * <tr><td class="z">
     * </td><td>                     .id=
     * </td><td class="z">                  String used for generating the query for results (i.e. proteins and global expression analysis <p />
     *                                      format:<type>[Like[S]{0,1}]{0,1}(<query>)[,<type>[Like[S]{0,1}]{0,1}(<query>)]{*} 
     * </td></tr>
     * <tr><td class="z">
     * </td><td>                     .full=
     * </td><td class="z">                  is shown as a description of the element <p />
     *                                      example: 5 Occureances;
     * </td></tr>
     * <tr><td class="z">      .total=
     * </td><td colspan = "2">       total number of suggestions <p />
     *                               necessary for select2 to know if there are more elements and therefore auto load them when scroll bar reaches the bottom
     * </td></tr>
     * </table>
     */	
    def suggest = {
        def query = (!params?.query)?'IPI':params.query     //user input for which the suggestions should be provided; this method only is called if length of query is 3 or greater;
        def max = (!params?.max)?10:params.max.toInteger()  //number of maximal shown entries in the list of suggestions; is set by initializing select2 (home.js for main page, find.js for "Proteins" or search.js for "Global expression analysis"
        def offset = (!params?.offset) ? 0 : (params.offset.toInteger()-1)*max      //offset for the list of suggestions; given by select2

        //res   list of suggestions
        //[:].elements[].type=type of the element; used for defining the class of the element in the list as well as if selected; 'gene' for gene name, 'uniprot' for protein name, 'go' for GO, 'ipi' for IPI identifier, 'description' for protein description
        //              .name=name of the elemnet; shown in list and if selected
        //              .id=String used for defining the query; is sent by clicking the submit button; format:<type>[Like[S]{0,1}]{0,1}(<query>)[,<type>[Like[S]{0,1}]{0,1}(<query>)]{*} 
        //              .full=is shown as a description of the element; example: 5 Occureances;
        //   .total=total number of suggestions; necessary for select2 to know if there are more elements and therefore load them when scroll bar reaches the bottom
        def res = queryService.getSuggestions(query, max, offset)      //get the suggestions

        //renders a linked hash map as JSON object; neccesary for select2
        def tmp = res as JSON
        def r = params.callback+'('+tmp.toString()+')'
        render r
    }

    /**
     * Generates a lists of results for "Global expression analysis" <p />
     * The action is called via AJAX and the results are rendered by GJP <p />
     * <p />
     * 
     * Parameter (params): <p />
     * 
     * <style>.custom-table td{border-top:1pt solid grey;vertical-align:top;} .custom-table{border-collapse:collapse;width:100%;} .custom-table .z{background-color:rgb(230, 230, 230);}</style>
     * <table class="custom-table">
     * <tr><td class="z">
     *              query
     * </td><td>        select2 generated query for proteins
     * </td></tr>
     * <tr><td class="z">
     *              max
     * </td><td>        maximal number of entries to be returned (pagination)
     * </td></tr>
     * <tr><td class="z">
     *              offset
     * </td><td>        offset number for pagination
     * </td></tr>
     * <tr><td class="z">
     *              type
     * </td><td>        specify which type of regulation should be returned <p />
     *                  'over/under' for up and down regulated, 'over' for up regulated, 'under' for down regulated, 'non-diff.expr.' for non differentially expressed
     * </td></tr>
     * <tr><td class="z">
     *              view
     * </td><td>        'Heatmap' for rendering as heatmap, 'List' for rendering as list
     * </td></tr>
     * <tr><td class="z">
     *              refLine
     * </td><td>        defines the tissue used as refference. 'avg' for using average of all (selected) tissues
     * </td></tr>
     * <tr><td class="z">
     *              [tissues]*
     * </td><td>        each tissue, which should go into the analysis has to be added individually
     * </td></tr>
     * </table>
     * 
     */	
    def globalExpressionResults = {
        def max = (params.max==null)?25:params.max.toInteger()      //number of elements in list; if scroll bar reaches the bottom the next are loaded; default is set to 25 if not set
        def offset = (params.offset==null)?0:params.offset.toInteger()      //offset of loaded elements in the list

        //define what kind of regulation is wanted and store it as boolean values
        boolean up = true
        boolean down = true
        boolean non = true
        if(params.type=='over/under'){
            non = false
        }else if(params.type=='over'){
            non = false
            down = false
        }else if(params.type=='under'){
            non = false
            up = false
        }else if(params.type=='non-diff.expr.'){
            up = false
            down = false
        }

        //CelllineGroup used as Reference or 'avg' for Average over all CelllineGroups
        def refLine = params.refLine.replace('_', ' ')

        Locale.setDefault(new Locale("US"));
        def start = System.currentTimeMillis()

        //List of Proteins maching the Query
        def proteinIds = queryService.getProteinIdsFromQuery(params.query, null)

        //get the wanted (queried) Celllines
        def celllines = []
        ExperimentClass.executeQuery('select distinct c.name from ExperimentClass c').each(){
            if(params[it.replace(' ', '_')]!=null){celllines.add(it)}
        }

        //clsCounts[projectId][CelllineGroup]=NrOfCelllinesInGroup
        def clsCounts = [:]
        ExperimentClass.executeQuery('select e.currentProject.id, e.currentClass.name, count(e.id) from Experiment e where e.id!=155 and e.id!=173 and e.currentClass.name in :cls group by e.currentProject.id, e.currentClass.name having count(e.id)>1', [cls:celllines]).each(){line->
            def proj = line[0]
            if(!clsCounts.containsKey(proj)){clsCounts.put(proj, [:])}
            clsCounts[proj].put(line[1], line[2])
        }

        //get Data from DB; data[projectId][ProteinId][CelllineGroup][CelllineId]=Intensity
        def data = queryService.getData(celllines, clsCounts.keySet() as List)
        
        //pValues[projectID][ProteinId]=pValue
        def pValues = queryService.generatePValues(data, celllines, clsCounts)

        //qValues[projectID][ProteinId]=qValue
        def qValues = queryService.generateQValues(pValues)
        
        def tmp_filteredData = queryService.filterData(data, proteinIds, qValues, refLine, up, non, down)
        def regulation = tmp_filteredData.regulation
        def nrOfFoundProts = tmp_filteredData.nrOfFoundProteins

        //pagination
        def nrOfProteins = regulation.size()
        def protIds = regulation.sort{it.order}*.protein
        def startPos = max*offset
        if(startPos<protIds.size()){
            def end = ((startPos+max)>=protIds.size())?protIds.size()-1:startPos+max-1
            protIds = protIds[startPos..end]
            regulation = regulation.sort{it.order}[startPos..end]
        }

        //get protein informations
        def resultProteins = [:]
        if(protIds.size()!=0){
            Protein.executeQuery('select p from Protein p where p.id in :prots', [prots:protIds]).each(){
                resultProteins.put(it.id, it)
            }
        }

        def projects = [:]
        Project.findAll().each{proj->
            projects.put(proj.id, proj.name)
        }
        def foundProts = [:]
        def enoughProts4PCA = true
        nrOfFoundProts.each{proj,prots->
            foundProts.put(projects[proj], prots)
            enoughProts4PCA = enoughProts4PCA&&(prots>1)
        }

        params.put('offset', offset+1)

        //render the results
        if(params.view=='List'){
            render(template:'list', model:[Proteins:resultProteins, Regulation:regulation.sort{it.order}, Celllines:celllines, Up:up, Down:down, Non:non, RefLine:refLine, Query:params.query, Type:params.type, FoundProteins:foundProts, ShowPCA:enoughProts4PCA, MaxEntries:max, Offset:offset, NrOfProteins:nrOfProteins, Params:params])
        }else if(params.view=='Heatmap'){
            render(template:'heatmap', model:[Proteins:resultProteins, Regulation:regulation.sort{it.order}, Celllines:celllines, Up:up, Down:down, Non:non, RefLine:refLine, Query:params.query, Type:params.type, FoundProteins:foundProts, ShowPCA:enoughProts4PCA, MaxEntries:max, Offset:offset, NrOfProteins:nrOfProteins, Params:params])
        }
    }
    
    
    /**
     * Generates a download file of ALL results for "Global expression analysis".<p />
     * The action is called via AJAX and streams the resulting file as response <p />
     * <p />
     * 
     * Parameter (params): <p />
     * 
     * <style>.custom-table td{border-top:1pt solid grey;vertical-align:top;} .custom-table{border-collapse:collapse;width:100%;} .custom-table .z{background-color:rgb(230, 230, 230);}</style>
     * <table class="custom-table">
     * <tr><td class="z">
     *              query
     * </td><td>        select2 generated query for proteins
     * </td></tr>
     * <tr><td class="z">
     *              type
     * </td><td>        specify which type of regulation should be returned <p />
     *                  'over/under' for up and down regulated, 'over' for up regulated, 'under' for down regulated, 'non-diff.expr.' for non differentially expressed
     * </td></tr>
     * <tr><td class="z">
     *              view
     * </td><td>        'Heatmap' for rendering as heatmap, 'List' for rendering as list
     * </td></tr>
     * <tr><td class="z">
     *              refLine
     * </td><td>        defines the tissue used as refference. 'avg' for using average of all (selected) tissues
     * </td></tr>
     * <tr><td class="z">
     *              [tissues]*
     * </td><td>        each tissue, which should go into the analysis has to be added individually
     * </td></tr>
     * </table>
     * 
     */	
    def globalExpressionDownload = {
        //define what kind of regulation is wanted and store it as boolean values
        boolean up = true
        boolean down = true
        boolean non = true
        if(params.type=='over/under'){
            non = false
        }else if(params.type=='over'){
            non = false
            down = false
        }else if(params.type=='under'){
            non = false
            up = false
        }else if(params.type=='non-diff.expr.'){
            up = false
            down = false
        }

        //CelllineGroup used as Reference or 'avg' for Average over all CelllineGroups
        def refLine = params.refLine.replace('_', ' ')

        Locale.setDefault(new Locale("US"));
        def start = System.currentTimeMillis()

        //List of Proteins maching the Query
        def proteinIds = queryService.getProteinIdsFromQuery(params.query, null)

        //get the wanted (queried) Celllines
        def celllines = []
        ExperimentClass.executeQuery('select distinct c.name from ExperimentClass c').each(){
            if(params[it.replace(' ', '_')]!=null){celllines.add(it)}
        }

        //clsCounts[projectId][CelllineGroup]=NrOfCelllinesInGroup
        def clsCounts = [:]
        ExperimentClass.executeQuery('select e.currentProject.id, e.currentClass.name, count(e.id) from Experiment e where e.id!=155 and e.id!=173 and e.currentClass.name in :cls group by e.currentProject.id, e.currentClass.name having count(e.id)>1', [cls:celllines]).each(){line->
            def proj = line[0]
            if(!clsCounts.containsKey(proj)){clsCounts.put(proj, [:])}
            clsCounts[proj].put(line[1], line[2])
        }

        //get Data from DB; data[projectId][ProteinId][CelllineGroup][CelllineId]=Intensity
        def data = queryService.getData(celllines, clsCounts.keySet() as List)

        //pValues[projectID][ProteinId]=pValue
        def pValues = queryService.generatePValues(data, celllines, clsCounts)

        //qValues[projectID][ProteinId]=qValue
        def qValues = queryService.generateQValues(pValues)
        
        //filter data and generate regulation
        def tmp_filteredData = queryService.filterData(data, proteinIds, qValues, refLine, up, non, down)
        def regulation = tmp_filteredData.regulation
        def nrOfFoundProts = tmp_filteredData.nrOfFoundProteins


        def protIds = regulation.sort{it.order}*.protein

        def resultProteins = [:]
        if(protIds.size()!=0){
            Protein.executeQuery('select p from Protein p where p.id in :prots', [prots:protIds]).each(){
                resultProteins.put(it.id, it)
            }
        }

        //preparing for returning a csv file
        response.setContentType("text/csv")
        response.setHeader("Content-disposition", "filename=${'Proteins.csv'}")
        
        //setting the first row of the csv table (column names)
        def head = 'Protein;Gene Names;Description;Location[Chromosome:Start-End:Strand];'
        celllines.each{cel->
            if(cel==refLine){
                head = head + cel+';'
            }else{
                if(up){head = head+'Up in '+cel+';'}
                if(down){head = head+'Down in '+cel+';'}
                if(non){head = head+'Non d.e. in '+cel+';'}
            }
        }
        
        //finding all projects and corresponding cell lines;
        def projects = [:]
        Project.findAll().each{proj->
            projects.put(proj.id, proj.name)
        }
        clsCounts.sort{it.key}.keySet().each{proj->
            head = head + 'p-Value in '+projects[proj]+';'
        }
        response.outputStream << head+'\n'
        
        //writing out the result of the analysis
        regulation.sort{it.order}.each(){p->
            //preparing on row of the file; set the protein identifier
            def line = resultProteins[p.protein].mainId+';'
            
            //add gene names komma separated
            resultProteins[p.protein].genes.eachWithIndex{gene,index->
                line = line + gene.name
                line = ((index+1)==resultProteins[p.protein].genes.size())?line:line+','
            }
            
            //adding protein description
            line = line + ';'+resultProteins[p.protein].description+';'
            
            //adding chromosome information
            resultProteins[p.protein].chromosomes.each{chrom->
                line = line + '[Chr. '+chrom.chromosome+':'+chrom.start+'-'+chrom.end+':'+chrom.strand+']'
            }
            line = line + ';'
            
            //adding number of projects regulated per tissue
            celllines.each{cel->
                if(cel==refLine){
                    line = line + 'Reference;'  //refference tissue has its own value
                }else{
                    def celData = (p.cls.containsKey(cel))?p.cls[cel]:[up:0, down:0, non:0]
                    if(up){line = line + celData.up+';'}
                    if(down){line = line + celData.down+';'}
                    if(non){line = line + celData.non+';'}
                }
            }
            
            //adding the corrected pValues of the projects
            clsCounts.sort{it.key}.keySet().each{proj->
                def projPValue = (p.projPValues.containsKey(proj))?p.projPValues[proj]:''
                line = line + projPValue+';'
            }
            
            //writing the row
            response.outputStream << line+'\n'
        }
        
        //returning the response
        response.outputStream.flush()
        return response
    }

    
    /**
     * Entry action for "Proteins" search option
     * This gives necessary information for the form (experiments). <p />
     * Could be called from main page, then 'showDefault' is set to true and query contains the in the main page setted query as string <p />
     * Further: <p />
     * The suggestions are generated by {@link #suggest}, called via AJAX, and rendered as JSON <p />
     * Querying the proteins is performed by {@link #proteinResults} for the web rendering and {@link #proteinDownload} for generating a file for download.
     */
    def proteins = {
        def showDefault = (params.showDefault!=null)?true:false     //only set as true iff redirected from first page
        def query = (params.query!=null)?params.query:''            //only set iff redirected;contains query from first page
        def jQuery = (showDefault && (query!=''))?queryService.getJQeryCallFromQuery(query):''      //foramts the query string to a JQuery data object coding string for adding the elements to select2 box 
        def projects = Project.findAll()
        
        return [Projects:projects, Query:query, ShowDefault:showDefault, JQuery:jQuery]
    }

    
    /**
     * Generates a list of results for "Proteins".<p />
     * The action is called via AJAX and the results are rendered by GJP <p />
     * <p />
     * 
     * Parameter (params): <p />
     * 
     * <style>.custom-table td{border-top:1pt solid grey;vertical-align:top;} .custom-table{border-collapse:collapse;width:100%;} .custom-table .z{background-color:rgb(230, 230, 230);}</style>
     * <table class="custom-table">
     * <tr><td class="z">
     *              query
     * </td><td>        select2 generated query for proteins. if not set or '', all proteins are listed
     * </td></tr>
     * <tr><td class="z">
     *              offset
     * </td><td>        offset for the number of queried entries (pagination) <p />
     *                  starts at 0. maximal entries are 10
     * </td></tr>
     * <tr><td class="z">
     *              [projectX]*
     * </td><td>        datasets to be shown <p />
     *                  X has to be replaced by the id of the project. for each project to be listed, this parameter has be set
     * </td></tr>
     * </table>
     * 
     */	
    def proteinResults = {
        def projects = []       //List of all selected experiments, which should be shown
        Project.findAll().each{proj->
            if(params['project'+proj.id]!=null){    //checks, if a project is included in the params
                projects.add(proj.id)
            }
        }
        def query = (params.query==null)?'':params.query    //sets default query to ''; this shows all proteins
        def offset = (params.offset==null)?0:params.offset.toInteger()      //sets default offset to 0
        def result = queryService.getProteinsFromQuery(query, projects, 10, offset)     //get the proteins

        def colors = helperService.getParameters('celllinecolors').celllinecolors     //get the colors for all tissues

        render(template:'proteinResults', model:[Proteins:result.proteins, Projects:projects, NrOfProteins:result.nrOfProteins, Colors:colors, Query:query, MaxEntries:10, Offset:offset])
    }

    
    /**
     * Generates a list of results for "Proteins".<p />
     * The action is called via AJAX and streams the resulting file as response <p />
     * <p />
     * 
     * Parameter (params): <p />
     * 
     * <style>.custom-table td{border-top:1pt solid grey;vertical-align:top;} .custom-table{border-collapse:collapse;width:100%;} .custom-table .z{background-color:rgb(230, 230, 230);}</style>
     * <table class="custom-table">
     * <tr><td class="z">
     *              query
     * </td><td>        select2 generated query for proteins. if not set or '', all proteins are listed
     * </td></tr>
     * <tr><td class="z">
     *              offset
     * </td><td>        offset for the number of queried entries (pagination) <p />
     *                  starts at 0. maximal entries are 10
     * </td></tr>
     * <tr><td class="z">
     *              [projectX]*
     * </td><td>        datasets to be shown <p />
     *                  X has to be replaced by the id of the project. for each project to be listed, this parameter has be set
     * </td></tr>
     * </table>
     * 
     */	
    def proteinDownload = {
        def query = (params.query==null)?'':params.query    //sets default query to ''; this shows all proteins
        
        //projects[ProjectId]=ProjectName   Maps all used project ids to its name   
        def projects = [:]
        //celllines[ProjectId][CelllineId]=CelllineName    Maps project ids to cell line ids to its cell line name
        def celllines = [:]
        
        //finding all projects and corresponding cell lines; stores them in projects and celllines
        Project.findAll().each{proj->
            if(params['project'+proj.id]!=null){    //checks, if a project is included in the params
                projects.put(proj.id, proj.name)
                celllines.put(proj.id, [:])
            }
        }
        LeadingProtein.executeQuery('select c.currentProject.id, c.id, c.name from Experiment c where c.currentProject.id in :proj order by c.name', [proj:(projects.keySet() as List)]).each{line->
            def proj = line[0]
            def cel = line[1]
            def celName = line[2]
            celllines[proj].put(cel, celName)
        }
        
        //preparing for returning a csv file
        response.setContentType("text/csv")
        response.setHeader("Content-disposition", "filename=${'Proteins.csv'}")
        
        //setting the first row of the csv table (column names)
        def header = 'Protein;Gene Names;Description'
        projects.each{pid,pname->
            celllines[pid].each{cid,cname->
                header = header+';'+pname+':'+cname
            }
        }
        response.outputStream << header+'\n'
        
        //get the queried protein information and append it to the file
        queryService.getAllProteinsFromQuery(query, projects.keySet() as List).each{protid,protdata->
            //preparing on row of the file; set the protein identifier
            def seq = protdata.mainId+';'
            
            //add gene names komma separated
            protdata.genes.eachWithIndex{g,i->
                seq = seq +((i==0)?'':',')+ g
            }
            
            //add the protein description
            seq = seq + ';' + protdata.description
            
            //for each project and each corresponding cell line: add the intensity; 0 if no value is given
            projects.each{pid,pname->
                def proj = (protdata.projects.containsKey(pid))?protdata.projects[pid]:[:]  //check if protein has values in one project; has to be a Map
                celllines[pid].each{cid,cname->
                    seq = seq + ';' + ((proj.containsKey(cid))?proj[cid]:0) //check if project data of a protein has intensities for a cell line
                }
            }
            
            //add one data row to the file
            response.outputStream << seq+'\n'
        }
        
        //return the csv file
        response.outputStream.flush()
        return response
    }
}
